{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import T5Tokenizer, T5ForConditionalGeneration\n",
    "\n",
    "tokenizer = T5Tokenizer.from_pretrained('mesolitica/t5-small-standard-bahasa-cased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# count = []\n",
    "# with open('shuffled-train.json') as fopen:\n",
    "#     for l in tqdm(fopen):\n",
    "#         data = json.loads(l)\n",
    "#         src = data['translation']['src']\n",
    "#         tgt = data['translation']['tgt']\n",
    "#         if 'promo' in src or 'promo' in tgt:\n",
    "#             count.append(data)\n",
    "            \n",
    "# count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['finetune-t5-small-standard-bahasa-cased-combined/checkpoint-680000',\n",
       " 'finetune-t5-small-standard-bahasa-cased-combined/checkpoint-690000',\n",
       " 'finetune-t5-small-standard-bahasa-cased-combined/checkpoint-700000']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from glob import glob\n",
    "\n",
    "checkpoints = sorted(glob('finetune-t5-small-standard-bahasa-cased-combined/checkpoint-*'))\n",
    "checkpoints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = T5ForConditionalGeneration.from_pretrained(checkpoints[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<pad> Hi guys! I noticed yesterday & today many people have got these cookies, right? So today I want to share some post mortem of our first batch:</s>\n"
     ]
    }
   ],
   "source": [
    "input_ids = tokenizer.encode('terjemah Melayu ke Inggeris: Hi guys! I noticed semalam & harini dah ramai yang dapat cookies ni kan. So harini i nak share some post mortem of our first batch:', return_tensors = 'pt')\n",
    "outputs = model.generate(input_ids, max_length = 100)\n",
    "print(tokenizer.decode(outputs[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<pad> mesolitica can make asr or not</s>\n"
     ]
    }
   ],
   "source": [
    "input_ids = tokenizer.encode('terjemah Melayu ke Inggeris: mesolitica boleh buat asr tak', return_tensors = 'pt')\n",
    "outputs = model.generate(input_ids, max_length = 100)\n",
    "print(tokenizer.decode(outputs[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<pad> Brother, I want to copy it on Facebook...hahaha. If not, I'll ss</s>\n"
     ]
    }
   ],
   "source": [
    "input_ids = tokenizer.encode('terjemah Melayu ke Inggeris: Bang, aku nak copy masuk kat fb…hahaha. Kalau xleh aku ss', return_tensors = 'pt')\n",
    "outputs = model.generate(input_ids, max_length = 100)\n",
    "print(tokenizer.decode(outputs[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<pad> Memang betul. Ini tidak perlu menjadi pakar, saya juga tahu. Ia adalah isyarat, bodoh.</s>\n"
     ]
    }
   ],
   "source": [
    "input_ids = tokenizer.encode(\"terjemah pasar Melayu ke Melayu: Memanglah. Ini tak payah expert, aku pun tau. It's a gesture, bodoh.\", return_tensors = 'pt')\n",
    "outputs = model.generate(input_ids, max_length = 100)\n",
    "print(tokenizer.decode(outputs[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<pad> Saya menulis untuk memohon jawatan Senior Software Engineer di [Company]. Sebagai seorang jurutera perisian yang sangat mahir dan berpengalaman dengan pengkhususan dalam seni bina data besar dan pemprosesan bahasa semula jadi, saya percaya saya akan menjadi tambahan berharga kepada pasukan anda</s>\n"
     ]
    }
   ],
   "source": [
    "input_ids = tokenizer.encode('terjemah Inggeris ke pasar Melayu: I am writing to apply for the Senior Software Engineer position at [Company]. As a highly skilled and experienced software engineer with a specialization in big data architecture and natural language processing, I believe I would be a valuable addition to your team', return_tensors = 'pt')\n",
    "outputs = model.generate(input_ids, max_length = 100)\n",
    "print(tokenizer.decode(outputs[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<pad> Jadi aku wasap mak aku yang aku sayang malam tu juga. Sambil tu aku tengah makan western food tu tapi.. aku tak sangka makanan tu sedap sebab aku tengah fikir nak settlekan benda yang ada dalam otak aku ni</s>\n"
     ]
    }
   ],
   "source": [
    "input_ids = tokenizer.encode(\"terjemah Inggeris ke pasar Melayu: So I smoked my mom whom I love that night too. Meanwhile, I was eating the western food but.. I didn't think that the food was good because I was thinking about settling the things that are in my brain\", return_tensors = 'pt')\n",
    "outputs = model.generate(input_ids, max_length = 100)\n",
    "print(tokenizer.decode(outputs[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['nous', 'can', 'do', 'asr', 'not?']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = 'nous bleh buat asr tak?'\n",
    "input_ids = [{'input_ids': tokenizer.encode(f'terjemah Melayu ke Inggeris: {s}', return_tensors='pt')[\n",
    "    0]} for s in t.split()]\n",
    "padded = tokenizer.pad(input_ids, padding='longest')\n",
    "outputs = model.generate(**padded, max_length = 50)\n",
    "tokenizer.batch_decode(outputs, skip_special_tokens=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<pad> asr</s>\n"
     ]
    }
   ],
   "source": [
    "input_ids = tokenizer.encode('terjemah Melayu ke Inggeris: asr', return_tensors = 'pt')\n",
    "outputs = model.generate(input_ids, max_length = 100)\n",
    "print(tokenizer.decode(outputs[0]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<pad> Is there a promo?</s>\n"
     ]
    }
   ],
   "source": [
    "input_ids = tokenizer.encode('terjemah Melayu ke Inggeris: ada promo x?', return_tensors = 'pt')\n",
    "outputs = model.generate(input_ids, max_length = 100)\n",
    "print(tokenizer.decode(outputs[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<pad> is there a promo?</s>\n"
     ]
    }
   ],
   "source": [
    "input_ids = tokenizer.encode('terjemah Melayu ke Inggeris: ada promo?', return_tensors = 'pt')\n",
    "outputs = model.generate(input_ids, max_length = 100)\n",
    "print(tokenizer.decode(outputs[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<pad> mesolitica can b</s>\n"
     ]
    }
   ],
   "source": [
    "input_ids = tokenizer.encode('terjemah Melayu ke Inggeris: mesolitica bleh b', return_tensors = 'pt')\n",
    "outputs = model.generate(input_ids, max_length = 100)\n",
    "print(tokenizer.decode(outputs[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<pad> got a promo?</s>\n"
     ]
    }
   ],
   "source": [
    "input_ids = tokenizer.encode('terjemah Melayu ke Inggeris: got promo?', return_tensors = 'pt')\n",
    "outputs = model.generate(input_ids, max_length = 100)\n",
    "print(tokenizer.decode(outputs[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Saya sedang menulis untuk memohon jawatan Senior Software Engineer di [ Kompany]. Sebagai seorang jurutera perisian yang sangat mahir dan berpengalaman dengan ada pengkhususan dalam seni bina data besar dan pemprosesan bahasa semula jadi, saya percaya saya akan menjadi tambahan berharga buat pasukan anda',\n",
       " 'Saya sedang menulis untuk memohon jawatan Jurutera Software Kanan di [ Kompany ]. Sebagai seorang jurutera perisian yang sangat mahir dan berpengalaman dengan khasisasi dalam seni bina data besar dan pemprosesan bahasa semula jadi, saya percaya saya akan menjadi tambahan yang berharga kepada pasukan anda',\n",
       " 'Saya menulis untuk memohon jawatan Senior Software Engineer di [ Syarikat ]. Sebagai seorang jurutera perisian yang berkemahiran tinggi dan berpengalaman dengan khasisasi dalam seni bina data besar dan pemprosesan bahasa semula jadi, saya percaya saya akan menjadi penambahan yang berharga kepada pasukan anda']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_ids = tokenizer.encode('terjemah Inggeris ke pasar Melayu: I am writing to apply for the Senior Software Engineer position at [Company]. As a highly skilled and experienced software engineer with a specialization in big data architecture and natural language processing, I believe I would be a valuable addition to your team', return_tensors = 'pt')\n",
    "outputs = model.generate(input_ids, max_length = 100, do_sample=True, \n",
    "    top_k=100, \n",
    "    top_p=0.95, temperature=0.7,\n",
    "    num_return_sequences=3)\n",
    "tokenizer.batch_decode(outputs, skip_special_tokens=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Rakyat mmg tak suka kau jugak',\n",
       " 'rakyat x suka ko pun',\n",
       " 'Rakyat mmg tak suka kau pon']"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_ids = tokenizer.encode(\"terjemah Melayu ke pasar Melayu: rakyat memang tak suka awak pun\", return_tensors = 'pt')\n",
    "outputs = model.generate(input_ids, max_length = 100, do_sample=True, \n",
    "    top_k=50, \n",
    "    top_p=0.95, \n",
    "                         temperature=0.7,\n",
    "    num_return_sequences=3)\n",
    "tokenizer.batch_decode(outputs, skip_special_tokens=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Perbincangan khas itu juga bertujuan utk Seri Paduka Baginda mendapat pandangan Raja2 Melayu bagi membolehkan baginda membuat keputusan terbaik demi dan kesejahteraan negara serta rakyat',\n",
       " 'Perbincangan khas itu juga bertujuan untuk Baginda mendapat pandangan Raja2 Melayu bagi membolehkan baginda membuat keputusan yang terbaik demi dan kesejahteraan negara dan rakyat',\n",
       " 'Perbincangan khas itu juga bertujuan agar Baginda mendapat pandangan Raja2 Melayu bagi membolehkan Baginda membuat keputusan yang terbaik demi kepentingan dan kesejahteraan negara serta rakyat']"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_ids = tokenizer.encode(\"terjemah Melayu ke pasar Melayu: Perbincangan khas itu juga bertujuan bagi Seri Paduka mendapat pandangan Raja-Raja Melayu untuk membolehkan baginda membuat keputusan yang terbaik demi kepentingan dan kesejahteraan negara serta rakyat\", return_tensors = 'pt')\n",
    "outputs = model.generate(input_ids, max_length = 100, do_sample=True, \n",
    "    top_k=100, \n",
    "    top_p=0.95,\n",
    "                         temperature=0.7,\n",
    "    num_return_sequences=3)\n",
    "tokenizer.batch_decode(outputs, skip_special_tokens=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['can', 'do', 'asr', 'no']"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = 'bleh buat asr tak'\n",
    "input_ids = [{'input_ids': tokenizer.encode(f'terjemah Melayu ke Inggeris: {s}', return_tensors='pt')[\n",
    "    0]} for s in t.split()]\n",
    "padded = tokenizer.pad(input_ids, padding='longest')\n",
    "outputs = model.generate(**padded, max_length = 50)\n",
    "tokenizer.batch_decode(outputs, skip_special_tokens=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<pad> can you do asr or not?</s>\n"
     ]
    }
   ],
   "source": [
    "input_ids = tokenizer.encode(f'terjemah Melayu ke Inggeris: {t}', return_tensors = 'pt')\n",
    "outputs = model.generate(input_ids, max_length = 100)\n",
    "print(tokenizer.decode(outputs[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"I don't understand\",\n",
       " 'Hi guys! I noticed yesterday & today many people have got these cookies, right? So today I want to share some post mortem of our first batch:',\n",
       " \"Indeed. This doesn't need to be an expert, I know too. It's a gesture, stupid.\",\n",
       " \"at 8 o'clock at the OKAY market, it's really crowded, he's good at choosing a place.\",\n",
       " \"So it's illegal\",\n",
       " 'where do you want to go?',\n",
       " \"It's like taking half a day\",\n",
       " \"Imagine Pakatan Harapan and win pru-14. After that there are all kinds of back doors. Last-last Ismail Sabri went up. That's why I pray not to give a fk about politics anymore. I swear it's up.\"]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strings = [\n",
    "    'ak tak paham la',\n",
    "    'Hi guys! I noticed semalam & harini dah ramai yang dapat cookies ni kan. So harini i nak share some post mortem of our first batch:',\n",
    "    \"Memanglah. Ini tak payah expert, aku pun tau. It's a gesture, bodoh.\",\n",
    "    'jam 8 di pasar KK memang org ramai 😂, pandai dia pilih tmpt.',\n",
    "    'Jadi haram jadah😀😃🤭',\n",
    "    'nak gi mana tuu',\n",
    "    'Macam nak ambil half day',\n",
    "    \"Bayangkan PH dan menang pru-14. Pastu macam-macam pintu belakang ada. Last-last Ismail Sabri naik. That's why I don't give a fk about politics anymore. Sumpah dah fk up dah.\",\n",
    "]\n",
    "input_ids = [{'input_ids': tokenizer.encode(f'terjemah Melayu ke Inggeris: {s}', return_tensors='pt')[\n",
    "    0]} for s in strings]\n",
    "padded = tokenizer.pad(input_ids, padding='longest')\n",
    "outputs = model.generate(**padded, max_length = 100)\n",
    "tokenizer.batch_decode(outputs, skip_special_tokens=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ini awak, bercakap dengan betul',\n",
       " 'baru menghadiri majlis perkahwinan sepupu saya. Pelik juga dia buat majlis biasa-biasa sebab gaya hidup dia nampak mewah. kemudian saya mendapat tahu mereka akan berbulan madu selama 3 minggu. keputusan pintar',\n",
       " 'Saya selepas melihat video ini: memang sedap burger benjo extra mayo',\n",
       " 'Hai semua! Saya perasan semalam & hari ni ramai yang dapat biskut ni kan? Jadi hari ini saya ingin berkongsi beberapa post mortem batch pertama kami:']"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strings = [\n",
    "    'u ni, talk properly lah',\n",
    "    \"just attended my cousin's wedding. pelik jugak dia buat majlis biasa2 je sebab her lifestyle looks lavish. then i found out they're going on a 3 weeks honeymoon. smart decision 👍\",\n",
    "    'Me after seeing this video: mm dapnya burger benjo extra mayo',\n",
    "    'Hi guys! I noticed semalam & harini dah ramai yang dapat cookies ni kan. So harini i nak share some post mortem of our first batch:',\n",
    "]\n",
    "input_ids = [{'input_ids': tokenizer.encode(f'terjemah pasar Melayu ke Melayu: {s}', return_tensors='pt')[\n",
    "    0]} for s in strings]\n",
    "padded = tokenizer.pad(input_ids, padding='longest')\n",
    "outputs = model.generate(**padded, max_length = 100)\n",
    "tokenizer.batch_decode(outputs, skip_special_tokens=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.push_to_hub('finetune-noisy-translation-t5-small-bahasa-cased-v4', organization='mesolitica')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "tokenizer.push_to_hub('finetune-noisy-translation-t5-small-bahasa-cased-v4', organization='mesolitica')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "_ = model.cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "5000it [07:05, 11.76it/s]\n"
     ]
    }
   ],
   "source": [
    "filtered_left, filtered_right = [], []\n",
    "\n",
    "with open('shuffled-test.json') as fopen:\n",
    "    for l in tqdm(fopen):\n",
    "        data = json.loads(l)['translation']\n",
    "        p = data['prefix']\n",
    "        src = data['src']\n",
    "        input_ids = [{'input_ids': tokenizer.encode(f'{p}: {src}', return_tensors = 'pt')[0]}]\n",
    "        padded = tokenizer.pad(input_ids, padding = 'longest')\n",
    "        for k in padded.keys():\n",
    "            padded[k] = padded[k].cuda()\n",
    "        outputs = model.generate(**padded, max_length = 256)\n",
    "        filtered_left.append(tokenizer.decode(outputs[0], skip_special_tokens=True))\n",
    "        filtered_right.append(data['tgt'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "refs = [filtered_right]\n",
    "sys = filtered_left"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sacrebleu.metrics import BLEU, CHRF, TER\n",
    "\n",
    "bleu = BLEU()\n",
    "chrf = CHRF(word_order = 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'name': 'BLEU',\n",
       " 'score': 64.06258219941243,\n",
       " '_mean': -1.0,\n",
       " '_ci': -1.0,\n",
       " '_verbose': '80.1/67.7/59.1/52.5 (BP = 1.000 ratio = 1.042 hyp_len = 111635 ref_len = 107150)',\n",
       " 'bp': 1.0,\n",
       " 'counts': [89388, 72166, 60115, 50792],\n",
       " 'totals': [111635, 106635, 101635, 96656],\n",
       " 'sys_len': 111635,\n",
       " 'ref_len': 107150,\n",
       " 'precisions': [80.07166211313655,\n",
       "  67.67571622825527,\n",
       "  59.14793132287106,\n",
       "  52.549246813441485],\n",
       " 'prec_str': '80.1/67.7/59.1/52.5',\n",
       " 'ratio': 1.0418572095193654}"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r = bleu.corpus_score(sys, refs)\n",
    "r.__dict__"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
